package prepro;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.HashSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class GenerateStopWords {
	public static void main(String[] args) throws Exception {
		BufferedReader brStop = new BufferedReader(new FileReader(new File(
				"../data/german_stop_words_sorted")));
		HashSet<String> stopWords = new HashSet<String>();
		BufferedReader brTerms = new BufferedReader(new FileReader(new File(
				"/local/joofeit/TermsLowerCapsUniq")));
		BufferedWriter bw = new BufferedWriter(new FileWriter(new File(
				"/local/joofeit/TermsLowerCapsUniqMinusStop")));

		boolean stopWord = false;
		Pattern p = Pattern
				.compile("^([a-z]|[A-Z]|ä|ü|ö|Ä|Ü|Ö|ß).*?([a-z]|[A-Z]|ä|ü|ö|Ä|Ü|Ö|ß)$");
		Pattern p2 = Pattern
				.compile("(„|\"|\')([a-z]|[A-Z]|ä|ü|ö|Ä|Ü|Ö|ß).*?([a-z]|[A-Z]|ä|ü|ö|Ä|Ü|Ö|ß)(\"|\')$");
		String line = "";
		int counter = 0;
		while ((line = brTerms.readLine()) != null) {

			Matcher m = p.matcher(line);
			Matcher m2 = p2.matcher(line);
			if (stopWords.contains((line.toLowerCase()))) {
				// System.out.println("Removed: "+word);
				System.out.println("1 >" + line + "< ");

			} else if (line.toLowerCase().startsWith("http")) {
				System.out.println("2 >" + line + "< ");

			} else if (line.trim().length() < 2) {
				System.out.println("3 >" + line + "< ");

			} else if (!m.matches()) {
				System.out.println("4 >" + line + "< ");

			} else {
				bw.write(line + "\n");
				counter++;
			}

		}

		brStop.close();
		brTerms.close();
		bw.close();
	}

}
